{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9ecc39b9",
   "metadata": {},
   "source": [
    "## 补课练习——招聘网站\n",
    "* 1.找到API数据的接口\n",
    "* 2.发送请求，请求到指定区域、职位的数据\n",
    "* 3.下载数据，存放到excel中\n",
    "* 4.数据分析：做该职位的地区分布（地图）、职位分布、职位标签（工作要求）、学历要求\n",
    "* 5.数据分析补充（待）\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5774225",
   "metadata": {},
   "source": [
    "## 期末项目\n",
    "* 数量500行x20列以上\n",
    "### 1.week13week14补课练习\n",
    "### 2.week15（selenium自动化模块）\n",
    "### 3.week01-week09（静态页面数据抓取）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c05f8ff7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from requests_html import HTMLSession\n",
    "import json\n",
    "import pandas as pd\n",
    "import requests\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b05005cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://www.zhipin.com/wapi/zpgeek/recommend/job/list.json?expectId=d3e657ff08678d3803Ry0t2-EFBX&experience=&degree=&scale=&stage=&welfare=&salary=&positionType=&payType=&multiBusinessDistrict=&multiSubway=&page=1&sortType=1\"\n",
    "payload = {\n",
    "    'expectId': 'd3e657ff08678d3803Ry0t2-EFBX',\n",
    "    'experience': '',\n",
    "    'degree': '',\n",
    "    'scale': '',\n",
    "    'stage': '',\n",
    "    'welfare': '',\n",
    "    'salary': '',\n",
    "    'positionType': '',\n",
    "    'payType': '',\n",
    "    'multiBusinessDistrict': '',\n",
    "    'multiSubway': '',\n",
    "    'page': 1,\n",
    "    'sortType': 1\n",
    "\n",
    "}\n",
    "\n",
    "# create an HTML session using Requests-HTML\n",
    "session = HTMLSession()\n",
    "\n",
    "# set the headers\n",
    "headers = {\n",
    "    'Accept': 'application/json, text/plain, */*',\n",
    "    'Accept-Encoding': 'gzip, deflate, br',\n",
    "    'Accept-Language': 'zh-CN,zh;q=0.9',\n",
    "#     'Cache-Control': 'no-cache',\n",
    "#     'Connection': 'keep-alive',\n",
    "#     'Content-Length': '412',\n",
    "#     'Content-Type': 'application/json;charset=UTF-8;',\n",
    "#     'Host':'we.51job.com',\n",
    "#     'Partner':'sem_pcbaiduyj_59565',\n",
    "#     'Property':'%7B%22partner%22%3A%22sem_pcbaiduyj_59565%22%2C%22webId%22%3A2%2C%22fromdomain%22%3A%2251job_web%22%2C%22frompageUrl%22%3A%22https%3A%2F%2Fwe.51job.com%2Fpc%2Fmy%2Fmyjob%22%2C%22pageUrl%22%3A%22https%3A%2F%2Fwe.51job.com%2Fpc%2Fsearch%22%2C%22identityType%22%3A%22%E5%9C%A8%E6%A0%A1%E7%94%9F%22%2C%22userType%22%3A%22%E8%80%81%E7%94%A8%E6%88%B7%22%2C%22isLogin%22%3A%22%E6%98%AF%22%2C%22accountid%22%3A%22227021599%22%7D',\n",
    "    # Cookie 是用户登录之后的cookie\n",
    "    'Cookie':'lastCity=101280100; wd_guid=18aad9d4-ca8a-46e1-8790-6d72a4a36033; historyState=state; YD00951578218230%3AWM_NI=xbAyb14zEVAx8TuNHOfooTdROCLNffPJ2z58o%2BiYPe2a75FPofrxC66F%2Bo2xzY33l3G9T5W47HlaD42Ws268fZCbAJ9cXgoUa6nGDLBpkxUD38n5nzW7lNuf%2Bzh4hicxUm0%3D; YD00951578218230%3AWM_NIKE=9ca17ae2e6ffcda170e2e6ee99ec4aadeaa48acb5df3ef8aa2d85f968a8fadd56eaab3a08de97293b5a4bbb62af0fea7c3b92abcbe8f9ae13bf8a68bb7c474fc8cbbafcb45a6b1a1b5e75a8d9d84afe56e9496fe9bb834adf5a7bad36ff2bff9d6c55cadbabe99c23ef69ae590b174ba87fdd4ef3985e79cd6d640f598a482c740f289bdd3b534a6868d83c442fbbd9888ee5fa3eaa2a8c74882b0a599d850f1e8a2d6ed7295e9a4aff86ee9a6a8d5eb63a1a89b8eb337e2a3; YD00951578218230%3AWM_TID=q9ayXfJASGpAFAFQERLALjnLxnyFklI7; _bl_uid=stlgyg9pa04xRXp55vtti0d0R58a; __zp_seo_uuid__=9da16ca0-f806-44b7-b250-b020ae6d91ba; __g=-; Hm_lvt_194df3105ad7148dcf2b98a91b5e727a=1685190236; Hm_lpvt_194df3105ad7148dcf2b98a91b5e727a=1685190248; __l=r=https%3A%2F%2Fwww.bing.com%2F&l=%2Fwww.zhipin.com%2Fweb%2Fgeek%2Fjob%3Fquery%3D%26city%3D101280100%26position%3D100901&s=3&g=&friend_source=0&s=3&friend_source=0; boss_login_mode=wechat; wt2=Df5RlBLgWsylCrOxLUx-EKtMY2DTUut66dVB1rQhVsuV2yWdoZ03B7_UExn0XybsRY179PCfXXRE8m4vWJWLCNA~~; wbg=0; __zp_stoken__=0933eIzI7Hzh4WEk3UHA1SBJjblA8BBJSBShBTB1YQysleXsrMWJ8Zw0jM09fDzt5fjtWP3A7cUkGOEIECh89CTcjdHJ9HhdfbB4dSzhoUDtHXBIOBlt3RAkgTk5hLxgyPAJkRHUbBUA2eT4%3D; __c=1685190236; __a=50470964.1667236966.1681137232.1685190236.12.3.6.12',\n",
    "    'Referer': 'https://www.zhipin.com/web/geek/recommend',\n",
    "    'Sec-Ch-Ua': '\"Google Chrome\";v=\"113\", \"Chromium\";v=\"113\", \"Not-A.Brand\";v=\"24\"',\n",
    "    'Sec-Ch-Ua-Mobile': '?0',\n",
    "    'Sec-Ch-Ua-Platform': '\"Windows\"',\n",
    "    'Sec-Fetch-Dest': 'empty',\n",
    "    'Sec-Fetch-Mode': 'cors',\n",
    "    'Sec-Fetch-Site': 'same-origin',\n",
    "    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36',\n",
    "    'Token':'BPhTnLURch19pbW3',\n",
    "    'Traceid':'0B0470C1-04B3-4F7E-B7AF-59874C4E258A',\n",
    "    'X-Requested-With':'XMLHttpRequest',\n",
    "    'Zp_token':'V1RN8nF-L-2lhjVtRvxxgZICu47jzUzCs~'\n",
    "\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0a45f8e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'code': 37, 'message': '您的访问行为异常.', 'zpData': {'name': '4ae04288', 'seed': 'lpqvQ/mfkELGbLZdNk/lizVo5h3jbAIxXPIslj7YYlA=', 'ts': 1685198675825}}\n"
     ]
    }
   ],
   "source": [
    "# send a POST request with headers\n",
    "r = session.get(url, data=json.dumps(payload), headers=headers)\n",
    "\n",
    "# extract the JSON data from the response\n",
    "response_data = r.json()\n",
    "\n",
    "# example: print the number of job postings returned\n",
    "print(response_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "87e1bb01",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'code': 37,\n",
       " 'message': '您的访问行为异常.',\n",
       " 'zpData': {'name': '4ae04288',\n",
       "  'seed': 'lpqvQ/mfkELGbLZdNk/lizVo5h3jbAIxXPIslj7YYlA=',\n",
       "  'ts': 1685198675825}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90217f67",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "\n",
    "driver = webdriver.Chrome()\n",
    "driver.get(\n",
    "    'https://www.zhipin.com/web/geek/job?query=%E6%B8%B8%E6%88%8F%E6%B5%8B%E8%AF%95%E5%B7%A5%E7%A8%8B%E5%B8%88&city=101280100&position=130111')\n",
    "lis = driver.find_elements_by_css_selector('.job-card-wrapper')\n",
    "for li in lis:\n",
    "    title = li.find_element_by_css_selector('.job-card-body clearfix').text\n",
    "    print(title)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ead9167",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import requests\n",
    "import json\n",
    "from lxml import etree\n",
    "import pandas as pd\n",
    "import pymongo\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium import webdriver\n",
    "dict_list=[]\n",
    "position_list=[]\n",
    "location_list=[]\n",
    "money_list=[]\n",
    "company_list=[]\n",
    "company_welfare_list=[]\n",
    "management_list=[]\n",
    "exp_list=[]\n",
    "detailed_url_list=[]\n",
    "browser=webdriver.Chrome()\n",
    "for j in range(1,11):\n",
    "    url='https://www.zhipin.com/c101040100/e_102/?query=python&page={}&ka=page-{}'.format(j,j)\n",
    "    browser.get(url)\n",
    "    time.sleep(5)\n",
    "    if j==1:\n",
    "        browser.find_element(By.//*[@id=\"header\"]/div[1]/div[3]/div/a[5],'//*[@id=\"header\"]/div[1]/div[3]/div/a[5]').click()\n",
    "        time.sleep(2)\n",
    "        browser.find_element(By.//*[@id=\"wrap\"]/div[2]/div[1]/div[2]/div[2],'//*[@id=\"wrap\"]/div[2]/div[1]/div[2]/div[2]').click()\n",
    "        time.sleep(5)\n",
    "        input=browser.find_element(By.//*[@id=\"container\"]/div[2]/div/div[1]/div/div[2]/div/div[1]/input,'//*[@id=\"container\"]/div[2]/div/div[1]/div/div[2]/div/div[1]/input')\n",
    "        input.click()\n",
    "        input.send_keys('python')\n",
    "        time.sleep(2)\n",
    "        browser.find_element(By.//*[@id=\"container\"]/div[2]/div/div[1]/div/div[2]/div/div[2]/ul/li[1],'//*[@id=\"container\"]/div[2]/div/div[1]/div/div[2]/div/div[2]/ul/li[1]').click()\n",
    "        time.sleep(3)\n",
    "        for i in range(1,31):\n",
    "            position=browser.find_element(By.//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[1]/a,'//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[1]/a'.format(i)).text\n",
    "            location=browser.find_element(By.//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[2]/span,'//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[2]/span'.format(i)).text\n",
    "            time.sleep(1)\n",
    "            money=browser.find_element(By.//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[2]/span,'//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[2]/span'.format(i)).text\n",
    "            company=browser.find_element(By.//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[2]/div/h3/a,'//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[2]/div/h3/a'.format(i)).text\n",
    "            time.sleep(1.5)\n",
    "            company_welfare=browser.find_element(By.//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[2]/div[2],'//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[2]/div[2]'.format(i)).text\n",
    "            management=browser.find_element(By.//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[2]/div/p/a,'//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[2]/div/p/a'.format(i)).text\n",
    "            time.sleep(1)\n",
    "            exp=browser.find_element(By.//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[2]/p,'//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[2]/p'.format(i)).text\n",
    "            detailed_url=browser.find_element(By.//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[1]/a.'//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[1]/a'.format(i)).get_attribute('href')\n",
    "            dict={'公司名称':company,'职位':position,'薪资待遇':money,'公司福利':company_welfare,'公司地点':location,'经营方向':management,'经验要求':exp,'联系它们网页':detailed_url}\n",
    "            dict_list.append(dict)\n",
    "            position_list.append(position)\n",
    "            location_list.append(location)\n",
    "            money_list.append(money)\n",
    "            company_list.append(company)\n",
    "            company_welfare_list.append(company_welfare)\n",
    "            management_list.append(management)\n",
    "            exp_list.append(exp)\n",
    "            detailed_url_list.append(detailed_url)\n",
    "            time.sleep(1)\n",
    "        print(dict_list)\n",
    "        print('第一页爬取完毕')\n",
    "    else:\n",
    "        for i in range(1,31):\n",
    "            position=browser.find_element(By.//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[1]/a,'//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[1]/a'.format(i)).text\n",
    "            location=browser.find_element(By.//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[2]/span,'//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[2]/span'.format(i)).text\n",
    "            time.sleep(1)\n",
    "            money=browser.find_element(By.//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[2]/span,'//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[2]/span'.format(i)).text\n",
    "            company=browser.find_element(By.//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[2]/div/h3/a,'//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[2]/div/h3/a'.format(i)).text\n",
    "            time.sleep(1.5)\n",
    "            company_welfare=browser.find_element(By.//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[2]/div[2],'//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[2]/div[2]'.format(i)).text\n",
    "            management=browser.find_element(By.//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[2]/div/p/a,'//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[2]/div/p/a'.format(i)).text\n",
    "            time.sleep(1)\n",
    "            exp=browser.find_element(By.//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[2]/p,'//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[2]/p'.format(i)).text\n",
    "            detailed_url=browser.find_element(By.//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[1]/a,'//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[1]/a'.format(i)).get_attribute('href')\n",
    "            #detailed_url = re.search(r'\\w+\\W\\w+\\.\\w+', str(detailed_url)).group(0)\n",
    "            #detailed_url = 'www.zhipin.com/' + detailed_url\n",
    "            dict={'公司名称':company,'职位':position,'薪资待遇':money,'公司福利':company_welfare,'公司地点':location,'经营方向':management,'经验要求':exp,'联系它们网页':detailed_url}\n",
    "            dict_list.append(dict)\n",
    "            position_list.append(position)\n",
    "            location_list.append(location)\n",
    "            money_list.append(money)\n",
    "            company_list.append(company)\n",
    "            company_welfare_list.append(company_welfare)\n",
    "            management_list.append(management)\n",
    "            exp_list.append(exp)\n",
    "            detailed_url_list.append(detailed_url)\n",
    "            time.sleep(1)\n",
    "        print(dict_list)\n",
    "        print('第{}页爬取完毕'.format(j))\n",
    "dict2={'公司名称':company_list,'职位':position_list,'薪资待遇':money_list,'公司福利':company_welfare_list,'公司地点':location_list,'经营方向':management_list,'经验要求':exp_list,'联系它们网页':detailed_url_list}\n",
    "dt=pd.DataFrame(dict2)\n",
    "dt.to_csv(r'C:\\Users\\asus\\Desktop\\boss直聘python.csv',mode='w')\n",
    "client=pymongo.MongoClient('mongodb://localhost:27017/')#链接mongodb数据库\n",
    "db=client['boss直聘python']#创建数据库\n",
    "db.collection.insert_one(dict2)#数据库中插入字典型数据dict2\n",
    "print('运行完毕')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f8b61fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver  # 操控浏览器 pip install selenium\n",
    "\n",
    "import csv  # 内置模块 不需要安装\n",
    "import time\n",
    "\n",
    "key_word = input('请输入你想要搜索的岗位名字: ') # print 输出 input 输入\n",
    "\n",
    "f = open(f'{key_word}data.csv', mode='a', encoding='utf-8', newline='')\n",
    "csv_writer = csv.DictWriter(f, fieldnames=['标题', '地区', '公司名字', '薪资',\n",
    "                                           '经验', '公司类型', '公司福利', '详情页'\n",
    "    ])\n",
    "\n",
    "csv_writer.writeheader()  # 写入表头\n",
    "\n",
    "driver = webdriver.Chrome()  # 实例化浏览器对象 我把驱动直接放在和python安装的同目录下面\n",
    "driver.get(f'https://www.zhipin.com/job_detail/?query={key_word}&city=100010000&industry=&position=')\n",
    "driver.implicitly_wait(10)  # 隐式等待 等浏览器数据加载完之后 就进行下一步操作\n",
    "# time.sleep(10) 死等 一定要等够10秒钟, 才进行下一步操作\n",
    "# 可以通过 css选择器 或者xpath 直接定位元素\n",
    "# 通过css选择器在元素面板里面查找\n",
    "# driver.find_element_by_css_selector('.ipt-search').send_keys('python') # 输入搜索内容\n",
    "# driver.find_element_by_css_selector('').click() # 点击\n",
    "# 通过两次提取, 第一次获取所有li标签内容\n",
    "\n",
    "def get_job_info():\n",
    "    lis = driver.find_elements_by_css_selector('.job-list li')\n",
    "    # 返回的是列表, 列表里面的元素 是 selenium对象\n",
    "    for li in lis:\n",
    "        area = li.find_element_by_css_selector('.job-area').text  # 地区\n",
    "        title = li.find_element_by_css_selector('.job-name a').text  # 标题\n",
    "        href = li.find_element_by_css_selector('.job-name a').get_attribute('href')  # 详情页\n",
    "        # 字符串替换replace()  remove 列表方法 strip() 去除字符串左右两端的空格\n",
    "        company_name = li.find_element_by_css_selector('.company-text .name a').get_attribute('title').replace('招聘',\n",
    "                                                                                                               '')  # 公司名字\n",
    "        money = li.find_element_by_css_selector('.job-limit .red').text  # 薪资\n",
    "        exp = li.find_element_by_css_selector('.job-limit p').text  # 经验学历\n",
    "        company_type = li.find_element_by_css_selector('.false-link').text  # 公司类型\n",
    "        company_boon = li.find_element_by_css_selector('.info-desc').text  # 公司福利\n",
    "        # ctrl + D 快速复制这行\n",
    "        # 创建一个字典, 把数据内容保存到字典里面\n",
    "        dit = {\n",
    "            '标题': title,\n",
    "            '地区': area,\n",
    "            '公司名字': company_name,\n",
    "            '薪资': money,\n",
    "            '经验': exp,\n",
    "            '公司类型': company_type,\n",
    "            '公司福利': company_boon,\n",
    "            '详情页': href,\n",
    "        }\n",
    "        csv_writer.writerow(dit)\n",
    "        print(title, area, company_name, money, exp, company_type, company_boon, href, sep='|')\n",
    "\n",
    "\n",
    "\n",
    "driver.quit()  # 自动关闭浏览器\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed6d1efd",
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install selenium==4.1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "505d6826",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import requests\n",
    "import json\n",
    "from lxml import etree\n",
    "import pandas as pd\n",
    "import pymongo\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium import webdriver\n",
    "dict_list=[]\n",
    "position_list=[]\n",
    "location_list=[]\n",
    "money_list=[]\n",
    "company_list=[]\n",
    "company_welfare_list=[]\n",
    "management_list=[]\n",
    "exp_list=[]\n",
    "detailed_url_list=[]\n",
    "browser=webdriver.Chrome()\n",
    "for j in range(1,11):\n",
    "    url='https://www.zhipin.com/c101040100/e_102/?query=python&page={}&ka=page-{}'.format(j,j)\n",
    "    browser.get(url)\n",
    "    time.sleep(5)\n",
    "    if j==1:\n",
    "        browser.find_element_by_xpath('//*[@id=\"header\"]/div[1]/div[3]/div/a[5]').click()\n",
    "        time.sleep(2)\n",
    "        browser.find_element_by_xpath('//*[@id=\"wrap\"]/div[2]/div[1]/div[2]/div[2]').click()\n",
    "        time.sleep(5)\n",
    "        input=browser.find_element_by_xpath('//*[@id=\"container\"]/div[2]/div/div[1]/div/div[2]/div/div[1]/input')\n",
    "        input.click()\n",
    "        input.send_keys('python')\n",
    "        time.sleep(2)\n",
    "        browser.find_element_by_xpath('//*[@id=\"container\"]/div[2]/div/div[1]/div/div[2]/div/div[2]/ul/li[1]').click()\n",
    "        time.sleep(3)\n",
    "        for i in range(1,31):\n",
    "            position=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[1]/a'.format(i)).text\n",
    "            location=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[2]/span'.format(i)).text\n",
    "            time.sleep(1)\n",
    "            money=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[2]/span'.format(i)).text\n",
    "            company=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[2]/div/h3/a'.format(i)).text\n",
    "            time.sleep(1.5)\n",
    "            company_welfare=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[2]/div[2]'.format(i)).text\n",
    "            management=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[2]/div/p/a'.format(i)).text\n",
    "            time.sleep(1)\n",
    "            exp=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[2]/p'.format(i)).text\n",
    "            detailed_url=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[3]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[1]/a'.format(i)).get_attribute('href')\n",
    "            dict={'公司名称':company,'职位':position,'薪资待遇':money,'公司福利':company_welfare,'公司地点':location,'经营方向':management,'经验要求':exp,'联系它们网页':detailed_url}\n",
    "            dict_list.append(dict)\n",
    "            position_list.append(position)\n",
    "            location_list.append(location)\n",
    "            money_list.append(money)\n",
    "            company_list.append(company)\n",
    "            company_welfare_list.append(company_welfare)\n",
    "            management_list.append(management)\n",
    "            exp_list.append(exp)\n",
    "            detailed_url_list.append(detailed_url)\n",
    "            time.sleep(1)\n",
    "        print(dict_list)\n",
    "        print('第一页爬取完毕')\n",
    "    else:\n",
    "        for i in range(1,31):\n",
    "            position=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[1]/a'.format(i)).text\n",
    "            location=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[2]/span'.format(i)).text\n",
    "            time.sleep(1)\n",
    "            money=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[2]/span'.format(i)).text\n",
    "            company=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[2]/div/h3/a'.format(i)).text\n",
    "            time.sleep(1.5)\n",
    "            company_welfare=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[2]/div[2]'.format(i)).text\n",
    "            management=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[2]/div/p/a'.format(i)).text\n",
    "            time.sleep(1)\n",
    "            exp=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[2]/p'.format(i)).text\n",
    "            detailed_url=browser.find_element_by_xpath('//*[@id=\"main\"]/div/div[2]/ul/li[{}]/div/div[1]/div[1]/div/div[1]/span[1]/a'.format(i)).get_attribute('href')\n",
    "            #detailed_url = re.search(r'\\w+\\W\\w+\\.\\w+', str(detailed_url)).group(0)\n",
    "            #detailed_url = 'www.zhipin.com/' + detailed_url\n",
    "            dict={'公司名称':company,'职位':position,'薪资待遇':money,'公司福利':company_welfare,'公司地点':location,'经营方向':management,'经验要求':exp,'联系它们网页':detailed_url}\n",
    "            dict_list.append(dict)\n",
    "            position_list.append(position)\n",
    "            location_list.append(location)\n",
    "            money_list.append(money)\n",
    "            company_list.append(company)\n",
    "            company_welfare_list.append(company_welfare)\n",
    "            management_list.append(management)\n",
    "            exp_list.append(exp)\n",
    "            detailed_url_list.append(detailed_url)\n",
    "            time.sleep(1)\n",
    "        print(dict_list)\n",
    "        print('第{}页爬取完毕'.format(j))\n",
    "dict2={'公司名称':company_list,'职位':position_list,'薪资待遇':money_list,'公司福利':company_welfare_list,'公司地点':location_list,'经营方向':management_list,'经验要求':exp_list,'联系它们网页':detailed_url_list}\n",
    "dt=pd.DataFrame(dict2)\n",
    "dt.to_csv(r'C:\\Users\\asus\\Desktop\\boss直聘python.csv',mode='w')\n",
    "client=pymongo.MongoClient('mongodb://localhost:27017/')#链接mongodb数据库\n",
    "db=client['boss直聘python']#创建数据库\n",
    "db.collection.insert_one(dict2)#数据库中插入字典型数据dict2\n",
    "print('运行完毕')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63789ba0",
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install pymongo"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
